#============================================================
# Create Time:			2020-11-04 12:02:53
# Last modify:			2020-11-04 12:43:37
# Writer:				Wenhao	1795902848@qq.com
# File Name:			idiom_download.py
# File Type:			PY Source File
# Tool:					Windows -- vim & python
# Information:			
#============================================================

import requests
from lxml import etree
import threading

lock = threading.Lock()
all_idoms = []

def get_page_urls():
    return ['http://www.rufengchengyu.com/list/2_%d.html'%(i+1) for i in range(296)]

def crawl_one_page(url):
    headers = {
            'Connection' : 'keep-alive',
            'Host' : 'www.rufengchengyu.com',
            'Referer' : 'http://www.rufengchengyu.com/list/2_296.html',
            'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36',
            }
    r = requests.get(url=url, headers=headers)
    raw_txt = r.content.decode('utf-8')
    html = etree.HTML(raw_txt)
    idioms = html.xpath('//div[@class="content"]//a/text()')
    return idioms


def run():
    def thread_todo():
        while(page_urls):
            lock.acquire()
            url = page_urls.pop()
            lock.release()

            idioms = crawl_one_page(url)

            lock.acquire()
            all_idoms.extend(idioms)
            lock.release()
            print('done %s'%(url))
        return None
    page_urls = get_page_urls()

    threads = [threading.Thread(target=thread_todo) for i in range(15)]

    for thread in threads:
        thread.start()

    for thread in threads:
        thread.join()

    with open('idioms.txt', 'w', encoding='utf-8') as f:
        print('saving data')
        f.write('\n'.join(all_idoms))
        print('done saving')

    return None

if __name__ == '__main__':
    run()
